# coding=utf-8
import sys
import random
import codecs


random.seed(222)

exp_file_tgt = []
# exp_file_tgt.append('fire_data/running/transformer-lm_bpe500_warmlr_format_pre2/27hyp.txt.1.topk20.t1.0')
# exp_file.append('fire_data/running/transformer-lm_bpe500_warmlr_format_pre2/27hyp.txt.1.topk10.t1.0')
# exp_file.append('out/bilstm_transformer-lm_bpe500_format_pre_srl_hier_pos_fix/17hyp.txt.1.topk20.t1.0')
# exp_file.append('out/bilstm_transformer-lm_bpe500_format_pre_srl_hier_pos_fix/17hyp.txt.1.topk10.t1.0')
# exp_file.append('out/bilstm_transformer-lm_bpe500_format_pre_srl_hier_pos_fix/17hyp.txt.1.topk5.t1.0')
# exp_file.append('out/bilstm_transformer-lm_bpe500_format_pre_srl_hier_pos_fix/17tgt.txt')
# exp_file.append('out/bilstm_transformer-lm_bpe500_format_pre_srl_hier_pos_fix/17hyp.txt.own.prompt2srl2_500_wo_smooth_d1_verb_loss.6.topk20.t1.0')
# exp_file.append('out/bilstm_transformer-lm_bpe500_format_pre_srl_hier_pos_fix/17hyp.txt.own.prompt2srl2_500_wo_smooth_d1_verb_loss.7.topk20.t1.0')
# exp_file.append('out/bilstm_transformer-lm_bpe500_format_pre_srl_hier_pos_fix/17hyp.txt.own.prompt2srl2_500_wo_smooth_d1_verb_loss.8.topk20.t1.0')
# exp_file.append('out/bilstm_transformer-lm_bpe500_format_pre_srl_hier_pos_fix/17hyp.txt.own.prompt2srl2_500_wo_smooth_d1_verb_loss.1.topk20.t1.0')
# exp_file_tgt.append('fire_data/running/transformer-lm_bpe500_warmlr_format_pre2_memory_gate_gpt/37hyp.txt.1.topk20.t1.0')
# exp_file_tgt.append('fire_data/running/transformer-lm_bpe500_warmlr_format_pre2_coherence_gpt/30hyp.txt.1.topk20.t1.0')
# exp_file_tgt.append('fire_data/running/transformer-lm_bpe500_warmlr_format_pre2_memory_gate_coherence_gpt/49hyp.txt.1.topk20.t1.0')
# exp_file_tgt.append('out/transformer-lm_bpe500_warmlr_format_pre2_memory_gate_relevance_gpt_fix/8hyp.txt.1.topk20.t1.0')
# exp_file_tgt.append('out/transformer-lm_bpe500_warmlr_format_pre2_memory_gate_relevance_gpt_fix/26hyp.txt.1.topk20.t1.0')

# event check
# exp_file.append('out/prompt2srl2_500/49tgt.txt')
# exp_file.append('out/prompt2srl2_500/49hyp.txt.1.topk10.t1.0.nr6')
# exp_file.append('out/prompt2srl2_500/49hyp.txt.1.topk20.t1.0.nr6')
# exp_file.append('out/prompt2srl2_500_wo_smooth_d1/40hyp.txt.1.topk20.t1.0.nr6')
# exp_file_tgt.append('fire_data/running/prompt2srl2_500_gpt2_nosympol/47tgt.txt')
# exp_file_tgt.append('fire_data/running/prompt2srl2_500_gpt2_nosympol/47hyp.txt.1.topk20.t1.0.nr6')
# exp_file_tgt.append('fire_data/running/prompt2srl2_500_wo_smooth_d1_gpt2_nosympol/12hyp.txt.1.topk20.t1.0.nr6')
# exp_file_tgt.append('fire_data/running/prompt2srl2_500_gpt2_nosympol/47tgt.txt')
# exp_file_tgt.append('fire_data/running/prompt2srl2_500_gpt2_nosympol/47hyp.txt.1.topk20.t1.0.nr6')
# exp_file_tgt.append('fire_data/running/prompt2srl2_500_gpt2_nosympol/47hyp.txt.11.topk20.t0.8.nr6')
# exp_file_tgt.append('fire_data/running/prompt2srl2_500_gpt2_nosympol/47hyp.txt.11.topk20.t0.6.nr6')
# exp_file_tgt.append('fire_data/running/prompt2srl2_500_gpt2_nosympol/47hyp.txt.11.topk20.t0.4.nr6')
# exp_file_tgt.append('fire_data/running/prompt2srl2_500_gpt2_nosympol/47hyp.txt.11.topk20.t0.2.nr6')
# exp_file_tgt.append('fire_data/running/prompt2srl2_500_wo_smooth_d1_gpt2_nosympol/12hyp.txt.1.topk20.t1.0.nr6')
# exp_file_tgt.append('fire_data/running/prompt2srl2_500_wo_smooth_d1_gpt2_nosympol/12hyp.txt.11.topk20.t0.8.nr6')
# exp_file_tgt.append('fire_data/running/prompt2srl2_500_wo_smooth_d1_gpt2_nosympol/12hyp.txt.11.topk20.t0.6.nr6')
# exp_file_tgt.append('fire_data/running/prompt2srl2_500_wo_smooth_d1_gpt2_nosympol/12hyp.txt.11.topk20.t0.4.nr6')
# exp_file_tgt.append('fire_data/running/prompt2srl2_500_wo_smooth_d1_gpt2_nosympol/12hyp.txt.11.topk20.t0.3.nr6')

# fire abs step check
# exp_file_tgt.append('fire_data/running/prompt2abs/14tgt.txt')
# exp_file_tgt.append('fire_data/running/prompt2abs/14hyp.txt.1.topk20.t1.0')
# exp_file_tgt.append('fire_data/running/prompt2abs/40hyp.txt.1.topk20.t1.0')

# second abs step check
# exp_file_tgt.append('fire_data/running/abs2story_small_ls0/41tgt.txt')
# exp_file_tgt.append('fire_data/running/abs2story_small_ls0/41hyp.txt.1.topk20.t1.0.abs')
# exp_file_tgt.append('fire_data/running/abs2story_small_ls0/41hyp.txt.1.topk20.t1.0')
# exp_file_tgt.append('fire_data/running/abs2story_small_ls0/41hyp.txt.own.prompt2abs.1.topk20.t1.0.abs')
# exp_file_tgt.append('fire_data/running/abs2story_small_ls0/41hyp.txt.own.prompt2abs.1.topk20.t1.0')

exp_file_tgt.append('fire_data/running/promptabsstory/45tgt.txt')
# exp_file_tgt.append('fire_data/running/transformer-lm_bpe500_warmlr_format_pre2/27hyp.txt.1.topk20.t1.0')
# exp_file_tgt.append('fire_data/running/promptabsstory/45hyp.txt.1.topk20.t1.0.abs')
# exp_file_tgt.append('fire_data/running/promptabsstory/45hyp.txt.1.topk20.t1.0')
# exp_file_tgt.append('fire_data/running/promptabsstory/45hyp.txt.own.prompt2abs.14hyp.txt.1.topk20.t1.0.for2input.1.topk20.t1.0.abs')
# exp_file_tgt.append('fire_data/running/promptabsstory/45hyp.txt.own.prompt2abs.14hyp.txt.1.topk20.t1.0.for2input.1.topk20.t1.0')
# exp_file_tgt.append('fire_data/running/promptabsstory/45hyp.txt.own.prompt2abs.40hyp.txt.1.topk20.t1.0.for2input.1.topk20.t1.0.abs')
# exp_file_tgt.append('fire_data/running/promptabsstory/45hyp.txt.own.prompt2abs.40hyp.txt.1.topk20.t1.0.for2input.1.topk20.t1.0')
# exp_file_tgt.append('fire_data/running/promptabsstory_mask_abs/18hyp.txt.1.topk20.t1.0.abs')
# exp_file_tgt.append('fire_data/running/promptabsstory_mask_abs/18hyp.txt.1.topk20.t1.0')
# exp_file_tgt.append('fire_data/running/promptabsstory_mask_abs/18hyp.txt.own.prompt2abs.14hyp.txt.1.topk20.t1.0.for2input.1.topk20.t1.0.abs')
# exp_file_tgt.append('fire_data/running/promptabsstory_mask_abs/18hyp.txt.own.prompt2abs.14hyp.txt.1.topk20.t1.0.for2input.1.topk20.t1.0')

# exp_file_tgt.append('fire_data/running/promptdstory/13hyp.txt.1.topk20.t1.0')

# exp_file_tgt.append('fire_data/running/promptabsstory_mask_abs_015_abs/38hyp.txt.1.topk20.t1.0.abs')
# exp_file_tgt.append('fire_data/running/promptabsstory_mask_abs_015_abs/38hyp.txt.1.topk20.t1.0')
# exp_file_tgt.append('fire_data/running/promptabsstory_mask_abs_015_abs/38hyp.txt.own.prompt2abs.14hyp.txt.1.topk20.t1.0.for2input.1.topk20.t1.0.abs')
# exp_file_tgt.append('fire_data/running/promptabsstory_mask_abs_015_abs/38hyp.txt.own.prompt2abs.14hyp.txt.1.topk20.t1.0.for2input.1.topk20.t1.0')
# exp_file_tgt.append('fire_data/running/promptabsstory_mask_abs_050_abs/23hyp.txt.1.topk20.t1.0.abs')
# exp_file_tgt.append('fire_data/running/promptabsstory_mask_abs_050_abs/23hyp.txt.1.topk20.t1.0')
# exp_file_tgt.append('fire_data/running/promptabsstory_mask_abs_050_abs/23hyp.txt.own.prompt2abs.14hyp.txt.1.topk20.t1.0.for2input.1.topk20.t1.0.abs')
# exp_file_tgt.append('fire_data/running/promptabsstory_mask_abs_050_abs/23hyp.txt.own.prompt2abs.14hyp.txt.1.topk20.t1.0.for2input.1.topk20.t1.0')

# exp_file_tgt.append('fire_data/running/prompt2abs/14hyp.txt.1.topk40.t1.0')
# exp_file_tgt.append('fire_data/running/prompt2abs/14hyp.txt.1.topk80.t1.0')
# exp_file_tgt.append('fire_data/running/prompt2abs/14hyp.txt.1.topk20.t2.0')
# exp_file_tgt.append('fire_data/running/prompt2abs/14hyp.txt.1.topk20.t1.0.nr6')

exp_file_tgt.append('fire_data/running/transformer-lm_bpe500_warmlr_format_pre2/27hyp.txt.1.topk20.t1.0')
exp_file_tgt.append('fire_data/running/transformer-lm_bpe500_warmlr_format_pre2_coref/26hyp.txt.1.topk20.t1.0')
exp_file_tgt.append('fire_data/running/promptdstory_discourse_two/13hyp.txt.1.min150.0.max300.topk20.t1.0')
exp_file_tgt.append('fire_data/running/promptdstory_discourse_two/29hyp.txt.1.min150.0.max300.topk20.t1.0')
exp_file_tgt.append('fire_data/running/promptdstory_discourse/8hyp.txt.1.min150.0.max300.topk20.t1.0')
exp_file_tgt.append('fire_data/running/promptdstory_discourse/30hyp.txt.1.min150.0.max300.topk20.t1.0')


base_path = '/home/rickwwang/project_research/'
surfix_path = '.human.test'

result_path = '/home/rickwwang/project_research/fairseq/data-bin/all_story.txt.test'

all_methods = []
for i, method in enumerate(exp_file_tgt):
    mid_path = exp_file_tgt[i]
    full_path = base_path + mid_path + surfix_path

    m1_file = codecs.open(full_path, 'r', encoding='utf8')
    m1_items = m1_file.readlines()

    topics = m1_items[::2]
    m1_items = m1_items[1::2]
    all_methods.append(m1_items)


def convert(x):
    x = [x[i].strip() + '\t' + str(i) for i in range(len(x))]
    # random.shuffle(x)
    return '\n'.join(x)


zip_all = zip(*all_methods)
print(len(zip_all))
items = map(convert, zip_all)
final = zip(topics, items)
final = filter(lambda x: x[0].strip() != 'null', final)
final = map(lambda x: ''.join(x), final)
print(len(final))

# write to file
with codecs.open(result_path, 'w', encoding='utf8') as fout:
    fout.write('\n'.join(final))
    fout.write('\n')
